{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Otto商品分类——线性SVM\n",
    "\n",
    "我们以Kaggle 2015年举办的Otto Group Product Classification Challenge竞赛数据为例，分别调用\n",
    "缺省参数LinearSVC、\n",
    "LinearSVC + CV进行参数调优（手动实现循环）。\n",
    "\n",
    "Otto数据集是著名电商Otto提供的一个多类商品分类问题，类别数=9. 每个样本有93维数值型特征（整数，表示某种事件发生的次数，已经进行过脱敏处理）。 竞赛官网：https://www.kaggle.com/c/otto-group-product-classification-challenge/data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "#竞赛的评价指标为logloss，但LinearSVC不支持概率\n",
    "#所以在这个例子中我们用正确率accuracy_score作为模型选择的度量\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 读取数据\n",
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "\n",
    "# 采用原始特征 + tf_idf特征\n",
    "#原始特征 + tf_idf特征对线性SVM训练还是很快，RBF核已慢得不行\n",
    "# RBF核只用tf_idf特征\n",
    "train1 = pd.read_csv(dpath +\"Otto_FE_train_org.csv\")\n",
    "train2 = pd.read_csv(dpath +\"Otto_FE_train_tfidf.csv\")\n",
    "#train = pd.read_csv(dpath +\"Otto_FE_train_tfidf.csv\")\n",
    "\n",
    "#去掉多余的id\n",
    "train2 = train2.drop([\"id\",\"target\"], axis=1)\n",
    "train =  pd.concat([train1, train2], axis = 1, ignore_index=False)\n",
    "train.head()\n",
    "\n",
    "del train1\n",
    "del train2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 将类别字符串变成数字\n",
    "# drop ids and get labels\n",
    "y_train = train['target']   #形式为Class_x\n",
    "X_train = train.drop([\"id\", \"target\"], axis=1)\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 训练样本6w+，交叉验证太慢，用train_test_split估计模型性能\n",
    "# SVM对大样本数据集支持不太好\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train, y_train, train_size = 0.8,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(49502, 186)\n"
     ]
    }
   ],
   "source": [
    "print (X_train_part.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 默认参数的 SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#LinearSVC不能得到每类的概率（只有predict函数，没有predict_proba函数），在Otto数据集要求输出每类的概率，这里只是示意SVM的使用方法\n",
    "#https://xacecask2.gitbooks.io/scikit-learn-user-guide-chinese-version/content/sec1.4.html\n",
    "#1.4.1.2. 得分与概率\n",
    "#1. 生成学习器实例\n",
    "SVC1 = LinearSVC()\n",
    "\n",
    "#2. 模型训练\n",
    "SVC1.fit(X_train_part, y_train_part)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('accuracy is\\xef\\xbc\\x9a ', 0.76430187459599219)\n",
      "Classification report for classifier LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0):\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "    Class_1       0.59      0.35      0.44       370\n",
      "    Class_2       0.65      0.86      0.74      3205\n",
      "    Class_3       0.52      0.35      0.41      1546\n",
      "    Class_4       0.76      0.15      0.25       566\n",
      "    Class_5       0.95      0.96      0.95       542\n",
      "    Class_6       0.93      0.92      0.93      2823\n",
      "    Class_7       0.69      0.62      0.66       572\n",
      "    Class_8       0.86      0.92      0.89      1703\n",
      "    Class_9       0.83      0.85      0.84      1049\n",
      "\n",
      "avg / total       0.76      0.76      0.75     12376\n",
      "\n",
      "\n",
      "Confusion matrix:\n",
      "[[ 129   16    3    0    1   41   10   77   93]\n",
      " [   3 2762  361   10   12   12   23   14    8]\n",
      " [   1  943  535    8    2    2   40    7    8]\n",
      " [   0  343   88   86    5   25   15    3    1]\n",
      " [   0   17    1    0  519    1    0    3    1]\n",
      " [  22   32    5    6    1 2609   46   54   48]\n",
      " [  16   71   35    1    3   42  357   41    6]\n",
      " [  19   24    7    0    4   42   17 1569   21]\n",
      " [  27   20    3    2    2   37   10   55  893]]\n"
     ]
    }
   ],
   "source": [
    "#3. 在校验集上测试，估计模型性能\n",
    "y_predict = SVC1.predict(X_val)\n",
    "\n",
    "print(\"accuracy is: \",accuracy_score(y_val, y_predict))\n",
    "\n",
    "print(\"Classification report for classifier %s:\\n%s\\n\"\n",
    "      % (SVC1, classification_report(y_val, y_predict)))\n",
    "print(\"Confusion matrix:\\n%s\" % confusion_matrix(y_val, y_predict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用原始特征 + tfidf特征的线性SVM分类性能：accuracy is： 0.76430187459599219\n",
    "\n",
    "class_1,class_3和class_4分类效果不好。\n",
    "是因为这几类样本数目少？（class_6类的样本数目也不多）。后面采用类别权重试试\n",
    "(用class_weight='balanced'效果更差了)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线性SVM正则参数调优"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "线性SVM LinearSVC的需要调整正则超参数包括C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） \n",
    "\n",
    "采用交叉验证，网格搜索步骤与Logistic回归正则参数处理类似，在此略。\n",
    "\n",
    "这里我们用校验集（X_val、y_val）来估计模型性能"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#单组超参数情况，模型在训练集上训练，在校验集上的测试的测试性能\n",
    "def fit_grid_point_Linear(C, X_train, y_train, X_val, y_val):\n",
    "    \n",
    "    # 在训练集上训练SVC\n",
    "    SVC2 =  LinearSVC( C = C)\n",
    "    SVC2 = SVC2.fit(X_train, y_train)\n",
    "    \n",
    "    # 在校验集上返回accuracy\n",
    "    accuracy = SVC2.score(X_val, y_val)\n",
    "    \n",
    "    print(\"C= {} : accuracy= {} \" .format(C, accuracy))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C= 0.1 : accuracy= 0.758322559793 \n",
      "C= 1.0 : accuracy= 0.764301874596 \n",
      "C= 10.0 : accuracy= 0.76567550097 \n",
      "C= 100.0 : accuracy= 0.763332255979 \n",
      "C= 1000.0 : accuracy= 0.7441822883 \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8lNX59/HPxa5FJawigqLFBVpF\njajFuuBSXIq4VMENq4jIktT+bLVPN7VP+7O1rQsgdcG1VbS4gCsuoGgrQhAUgarI44KgBgRRZBG4\nnj/OnWYMCZlkcs89k/m+X695JTNzZuaageSb+5xzn2PujoiISH01SboAERHJbwoSERHJiIJEREQy\noiAREZGMKEhERCQjChIREcmIgkRERDKiIBERkYwoSEREJCPNki4gG9q3b++777570mWIiOSVOXPm\nrHD3DrW1K4gg2X333SkrK0u6DBGRvGJm76fTTl1bIiKSEQWJiIhkREEiIiIZUZCIiEhGFCQiIpIR\nBYmIiGREQSIiIhkpiPNIRHLJqlWwYAEsXAhffQXHHAPf+Q6YJV2ZSP0oSERi8tlnlYGR+vXjj7du\n26UL9O8PJ5wQgqVNm+zXK1JfChKRDK1cWX1gfPJJZZtvfQt69gxh0bMn9OoVvjZrBs88A089BZMm\nwYQJ0LQpfO97lcGy//7QRJ3QksPM3ZOuIXbFxcWuJVIkUytWVB8Yn35a2aZ1628GRcXXrl1rD4NN\nm2DmTHj66RAsr70Wbu/UKYRK//5w/PHQtm1871EklZnNcffiWtvFGSRm1h+4EWgK3O7u11a5/3rg\n6Ojq9kBHd28T3dcNuB3oCjhworu/Z2Z3AUcCn0ePu8Dd522rDgWJ1EV5efWBUV5e2WaHHWoOjIYa\n6/jkE5g6NQTL1Kmhq6xJE+jTJxyp9O8PxcU6WpH4JB4kZtYUeBs4DlgKzAYGu/vCGtqPBg5w9wuj\n6y8Av3f3Z82sNbDF3b+KguRxd5+Ubi0KEqnKvebAWLGist2OO1aGRGpg7LprdgfHN2+GsrJwpPL0\n0zBrVngP7duHo5QTTghfO3bMXk3S+KUbJHGOkfQBFrv7kqigicApQLVBAgwGfhu17Qk0c/dnAdz9\nyxjrlEbMPXQ9VRcYK1dWtttppxASAwd+8yhjl11yYzZV06ZwyCHhctVVIeyefTYEy9SpcN99oV1x\nceXYSp8+YQxGJG5xHpGcAfR396HR9fOAQ9x9VDVtdwNmAru6+2YzGwgMBTYC3YHngCuj++4CDgM2\nAM9Ht2+o5jmHAcMAunXrdtD776e1GrLkKffQFVRdYHz2WWW7Nm227o7q1Qs6d86NwKiPLVtg7tzK\nsZVXXgm3FRXBccdVjq907px0pZJvcqFr60fAD6oESR93H11N2ysIITI6un4GMAE4APgAeAB40t0n\nmFln4GOgBXAr8K67X7OtWtS11Xi4h+mz1QXGqlWV7YqKqg+MnXfO38BI16pV8NxzlcGyfHm4ff/9\nK8dWvvc9aN482Tol9+VC19ZSwkB5hV2BZTW0HQSMrPLYuSndYo8ChwIT3D36sWCDmd0JXN6gVUtO\ncA+/AKsGxsKF3wyMtm1DQJx55jcDo1Onxh8YNSkqgh/9KFzcYf78yrGVP/8Zrr02TBY49tjKYOna\ntfbnFalJnEEyG+hhZt2BjwhhcXbVRma2N1AEvFLlsUVm1sHdy4F+QFnUvrO7LzczAwYCb8b4HiRm\n7rBsWfWBsXp1Zbt27UJAnHXWNwOjY8fCDYx0mMF++4XLFVfAmjUwbVoIlqeegkceCe169aocWzn8\ncGjZMtm6Jb/EPf33ROAGwvTfO9z992Z2DVDm7lOiNlcBrdz9yiqPPQ74C2DAHGCYu280s2lAh+j2\necDw2gbj1bWVPHf46KPqA+PzzyvbtW9ffZdUhw4KjIbmDosWVXaBzZgBGzeGkyf79asMlu7dk65U\nkpL4GEkuUZBkjzssXVp9YKxZU9muQ4eaA0OSsXYtTJ9eGSxLloTb99qrsgvsyCNhu+2SrVOyR0GS\nQkESv6++gp//HO65B774ovL2jh23DoyePRUYuc4dFi+uHFuZPh3Wr4dWreCooyqDpUcPHSk2ZgqS\nFAqSeL3xBgweHI46hgyBQw+tDIz27ZOuThrCunWh66siWN56K9y+xx6VXWBHHx26xaTxUJCkUJDE\nwx3GjoWf/SzMFLr33jATSBq/JUvCiZBPPRUG79euhRYt4IgjKoNl3311tJLvFCQpFCQNr7wcLrwQ\nHn8cTj4Z7rhD3VWFasMGePnlyrGVBQvC7d26VZ4MecwxYbkZyS8KkhQKkob13HNw/vnhjPE//xlG\njtRfnlLpww9DqDz9dFjG5YsvwlItfftWjq3st5/+z+QDBUkKBUnD2LgRfv1ruO462GcfmDgx/EIQ\nqcnXX4clWyrGVuZF63R37lzZBXbssaFrVHKPgiSFgiRzixeHAfWyMrjkEvjrX2H77ZOuSvLN8uWV\nYyvPPBNOOm3aNEzQqAiWAw7Q0vi5QkGSQkFSf+5hEH3kyLA20+23w2mnJV2VNAabNoXl8CvGVip+\nRDt2hB/8oHIjL838S46CJIWCpH7WrIFLLw1LlB95ZAgUrckkcfn003CUUrGR14oVYRzl4IMrx1YO\nPjgcwUh2KEhSKEjqbuZMOPts+OCDsP/FL36hH2DJns2bw1bDFWMrr74alsb/9rfDkctOOyVdYWFI\nN0jUEynfsHkz/OEPYeE+d3jpJfjVrxQikl1Nm4ajj9/8Bv7973C0csstYazuzjuTrk6qUpDIf330\nUZhB88tfhiXI582Dww5LuiqRsPrzsGFhCvGYMeEPHskdChIBYPLkMJV39uzwF99996n7QHJPSUk4\nq/7JJ5OuRFIpSArcunUwYkTYq7x799AvfcEFOllMctOpp8Kuu8KNNyZdiaRSkBSw+fNDP/T48XD5\n5aEveq+9kq5KpGbNm4ep6M8/X7kUiyRPQVKA3GHcuBAiK1aEqZbXXRcW3RPJdRdfHJazv+mmpCuR\nCgqSArNiRejGGjUqLKT3xhvhpC+RfNGuHZx7bjiv6bPPkq5GQEFSUKZNg/33D/Pyb7ghrNzbsWPS\nVYnUXUlJGN+7/fakKxFQkBSEr78OJxQee2xYyvvVV6G0VAPqkr+++92wkdbYsWGpFUmWgqSRe/fd\ncHLhtdfC0KHhrODevZOuSiRzpaVhyfrJk5OuRBQkjdjf/x5WUn37bfjnP+HWW7UVqjQeJ58cpqxr\nKnDyFCSN0Jo1cN554dK7N7z+OpxxRtJViTSspk3DpJGXXoK5c5OuprApSBqZWbPCUch998HVV4cB\n9m7dkq5KJB4XXhiOsjUVOFkKkkZiy5YwDtK3bxh8nDEjLHjXrFnSlYnEp00bGDIk/OH06adJV1O4\nFCSNwLJlcNxxYWbWaaeFrqy+fZOuSiQ7Ro8O20DfemvSlRQuBUmee+yxsNjizJkwYULYR71Nm6Sr\nEsmeffYJOyrefHMIFMk+BUmeWrcuDDQOGBDGQF57LfQX69wQKUSlpWE/+IceSrqSwqQgyUMLFkCf\nPmG9rJ/+FF55BfbeO+mqRJLzgx+EBUc1FTgZCpI84h5W6i0uDgOLTz0Ff/kLtGyZdGUiyWrSJIyV\nvPpquEh2KUjyxMqVYSB9xAg46qiw2GL//klXJZI7hgwJSwBpKnD2KUjywAsvhMUWn3gC/vrX8LVT\np6SrEsktO+wQxgkffDDMZJTsUZDksK+/Dvun9+sHrVuHQ/bLLguH8SKytVGjwn7u48cnXUlh0a+k\nHLVkCRxxBPzhD+GvrDlzwhnrIlKzPfcMa3DdcgusX590NYVDQZKD7rsvrJG1aBE88EDYc0GLLYqk\np7QUysvDOVWSHQqSHPLFF2HA8Jxzwn4Lr78OZ56ZdFUi+aVfP+jVKwy6uyddTWFQkOSIsjI48MCw\n9PtvfgMvvgi77ZZ0VSL5xyzsoDh3Lrz8ctLVFAYFScK2bIE//QkOOww2bAgztK6+WostimTi3HOh\nqEgnKGZLrEFiZv3N7C0zW2xmV1Zz//VmNi+6vG1mq1Pu62Zmz5jZIjNbaGa7R7d3N7NXzewdM3vA\nzFrE+R7itHx5OCP3iitg4MDQlfX97yddlUj+2357uPhieOQR+OCDpKtp/GILEjNrCowDTgB6AoPN\nrGdqG3e/zN17u3tvYAzwcMrd9wDXufu+QB+gYpHoPwLXu3sPYBVwUVzvIU6PPx4WW/zXv+C228Lc\n96KipKsSaTxGjgzdXOPGJV1J4xfnEUkfYLG7L3H3jcBE4JRttB8M3A8QBU4zd38WwN2/dPevzMyA\nfsCk6DF3AwPjegNxWL8+9N/+8IfQpUuY1jt0qBZbFGlo3brBqaeGP9S++irpahq3OIOkC/BhyvWl\n0W1bMbPdgO7AtOimvYDVZvawmc01s+uiI5x2wGp335TGcw4zszIzKysvL2+At5O5hQvhkENgzBj4\nyU/C0u/77pt0VSKNV0kJrFoVJrFIfOIMkur+xq5pMt4gYJK7b46uNwO+D1wOHAzsAVxQl+d091vd\nvdjdizt06FCXuhucezhBqrg4jIs88QRcfz20apVoWSKN3uGHhxN5NRU4XnEGyVKga8r1XYGaVsAZ\nRNStlfLYuVG32CbgUeBAYAXQxswq5jRt6zlzwmefwRlnwPDh4T/1G2/AiScmXZVIYaiYCrxgAUyb\nVnt7qZ84g2Q20COaZdWCEBZTqjYys72BIuCVKo8tMrOKQ4l+wEJ3d2A6cEZ0+xBgckz1Z+zFF8Ni\ni489BtddB08/DTvvnHRVIoVl0CDo0EFTgeMUW5BERxKjgKnAIuBBd19gZteY2YCUpoOBiVFIVDx2\nM6Fb63kzm0/o0rotuvsK4KdmtpgwZjIhrvdQX5s2hZMK+/WD7bYLG09dfrkWWxRJQqtWoUfg8cfh\n3XeTrqZxMi+AjsPi4mIvKyvLymu99x6cfXYIjx//OPTNtm6dlZcWkRosWxZWihg1KoxPSnrMbI67\nF9fWTn8jN6CJE0NX1oIFcP/9cMcdChGRXLDLLmHdujvuCGvaScNSkDSAL78MRx+DB4fF4ubNC/2y\nIpI7SkpgzRq4++6kK2l8FCQZeu21sNji3XfDr34FM2ZA9+5JVyUiVR1ySLjcdFNY404ajoKknrZs\ngb/8BQ49FNatg+nT4Xe/02KLIrmstBTeeQemTk26ksZFQVIPH38MJ5wQZmKdfHJYbPHII5OuSkRq\nc/rp0LmzpgI3NAVJHT35ZFhs8aWX4G9/g4cegrZtk65KRNLRogWMGBGOSP7zn6SraTwUJGnasAEu\nuwxOOimcVFhWBpdcosUWRfLNsGEhUMaMSbqSxkNBkoZFi8Ig3Q03wOjRMGsW9OxZ++NEJPd07BjO\n9br7bli9uvb2UjsFyTa4hyWoDzoIPvooLHVy001abFEk35WUwNq14bwSyZyCpAbucP754TC4b9+w\n2OLJJyddlYg0hAMOCLuRjhkDmzfX3l62TUFSA7Owj/of/xgG5jp3TroiEWlIpaVhSaPHHku6kvyn\ntbZEpCBt2gR77hkuWmK+elprS0RkG5o1C/u6T58euq6l/hQkIlKwhg4NWz1oKnBmFCQiUrDatoXz\nzgt7uq9YkXQ1+UtBIiIFraQE1q8PU/2lfhQkIlLQevWCY46Bm2+Gr79Oupr8pCARkYJXWgpLl8Ij\njyRdSX5SkIhIwTvppDAN+Kabkq4kPylIRKTgNWkS9nP/179gzpykq8k/ChIREcJ22a1ba6+S+kgr\nSMzsITM7ycwUPCLSKO20E1xwAUycGDavk/SlGwzjgbOBd8zsWjPbJ8aaREQSMXp0mLl1yy1JV5Jf\n0goSd3/O3c8BDgTeA541s3+b2Y/NrHmcBYqIZMtee8GJJ8L48bBxY9LV5I+0u6rMrB1wATAUmAvc\nSAiWZ2OpTEQkASUl8Mkn8OCDSVeSP9IdI3kYeAnYHvihuw9w9wfcfTTQOs4CRUSy6fjjYZ99wqB7\nASyO3iDSPSIZ6+493f1/3X156h3pLDEsIpIvzMJYSVkZzJyZdDX5Id0g2dfM2lRcMbMiMxsRU00i\nIok6//wwi0tTgdOTbpBc7O6rK664+yrg4nhKEhFJVuvWYYn5SZPC0imybekGSRMzs4orZtYUaBFP\nSSIiyRs5MoyRjB+fdCW5L90gmQo8aGbHmFk/4H7g6fjKEhFJVvfuMGBAOKdk3bqkq8lt6QbJFcA0\n4FJgJPA88PO4ihIRyQUlJbByJdx/f9KV5DbzApjfVlxc7GVlZUmXISJ5xh169w7fz5sXZnQVEjOb\nk87M3HTPI+lhZpPMbKGZLam4ZF6miEjuMgtHJW+8ATNmJF1N7kq3a+tOwnpbm4CjgXuAe+MqSkQk\nV5x9NrRrp6nA25JukGzn7s8TusLed/ergH7xlSUikhu22w6GDYPJk+G995KuJjelGyTroyXk3zGz\nUWZ2KtAxxrpERHLGpZeGbq5x45KuJDelGyQ/IayzVQIcBJwLDKntQWbW38zeMrPFZnZlNfdfb2bz\nosvbZrY65b7NKfdNSbn9LjP7fyn39U7zPYiI1EvXrnD66XD77bB2bdLV5J5mtTWITj48091/BnwJ\n/DidJ44eNw44DlgKzDazKe6+sKKNu1+W0n40cEDKU6xz95pC4mfuPimdOkREGkJpaVgR+N57Yfjw\npKvJLbUekbj7ZuCg1DPb09QHWOzuS9x9IzAROGUb7QcTTnQUEck5hx0GBx0EN92kVYGrSrdray4w\n2czOM7PTKi61PKYL8GHK9aXRbVsxs92A7oSTHiu0MrMyM5tpZgOrPOT3ZvZG1DXWsobnHBY9vqy8\nvLyWUkVEts0sHJUsWgTPahemb0g3SNoCKwkztX4YXU6u5THVHcHUlOODgEnR0U+FbtGJMGcDN5jZ\nntHtvwD2AQ6O6rqiuid091vdvdjdizt06FBLqSIitTvzTOjUKRyVSKVax0gA3D2tcZEqlgJdU67v\nCiyroe0gwtIrqa+5LPq6xMxeIIyfvJuyH8oGM7sTuLwetYmI1FnLlmF85Oqr4Z13oEePpCvKDeme\n2X6nmd1R9VLLw2YDPcysu5m1IITFlKqNzGxvoAh4JeW2ooouKzNrD/QFFkbXO0dfDRgIvJnOexAR\naQjDh0Pz5jB2bNKV5I50u7YeB56ILs8DOxJmcNXI3TcBowgrBy8CHnT3BWZ2jZkNSGk6GJjo31z0\na1+gzMxeB6YD16bM9vqHmc0H5gPtgf+b5nsQEcnYzjvDWWfBnXfCmjVJV5Mb6rVoY3Ry4nPunhdn\nt2vRRhFpSLNnQ58+cMMNYQC+sWrQRRur0QPoVs/HiojktYMPDtOBx4yBLVuSriZ56Y6RfGFmayou\nwGPUMFtKRKQQlJbCu+/Ck08mXUny0p21tUPchYiI5JPTToMuXcJU4JNrOxmikUv3iORUM9sp5Xqb\nak4SFBEpGM2bw4gR4eTEhQtrb9+YpTtG8lt3/7ziiruvBn4bT0kiIvlh2LBwbkmhn6CYbpBU1y6t\nbjERkcaqfXs45xy45x5YtSrpapKTbpCUmdlfzWxPM9vDzK4H5sRZmIhIPigthXXrwhLzhSrdIBkN\nbAQeAB4E1lFlSRMRkUK0335w1FHhTPdNm5KuJhlpBYm7r3X3KysWQXT3/+Pu2t5FRAQoKYEPPoAp\nWy0CVRjSnbX1rJm1SbleZGZT4ytLRCR/DBgAu+0GN96YdCXJSLdrq300UwsAd1+F9mwXEQGgaVMY\nNQpmzIB585KuJvvSDZItZvbfJVHMbHdq3ltERKTgXHQRbL99YU4FTjdIfgm8bGb3mtm9wIuEDaZE\nRAQoKoIhQ+C++6DQNmVNd7D9aaAYeIswc+t/CDO3REQkMno0bNgAt96adCXZle5g+1DCPiT/E13u\nBa6KrywRkfyz775w/PFw883w9ddJV5M96XZtlRL2SH/f3Y8mbHtbYAdvIiK1KymBZcvgoYeSriR7\n0g2S9e6+HsDMWrr7f4C94ytLRCQ/nXBC2Mu9kKYCpxskS6PzSB4FnjWzycCy+MoSEclPTZqEsZKZ\nM2HWrKSryY50B9tPdffV7n4V8GtgAqBl5EVEqjFkCOywQ+FMBa7zVrvu/qK7T3H3jXEUJCKS73bc\nES68EB58EJYvT7qa+NV3z3YREdmGUaPCIo5/+1vSlcRPQSIiEoNvfxtOOikEyYYNSVcTLwWJiEhM\nSkvh009h4sSkK4mXgkREJCbHHAM9e4apwN6IVydUkIiIxMQsnKA4dy78619JVxMfBYmISIzOPRfa\ntGncU4EVJCIiMfrWt+Dii+Hhh+HDD5OuJh4KEhGRmI0cGcZIxo1LupJ4KEhERGK2224wcCDcdht8\n9VXS1TQ8BYmISBaUlsJnn8E//pF0JQ1PQSIikgXf/z7sv38YdG9sU4EVJCIiWWAWjkrefBOmT0+6\nmoalIBERyZLBg6F9+8a3V4mCREQkS1q1gksugccegyVLkq6m4ShIRESy6NJLoWlTGDs26UoajoJE\nRCSLunSBM86ACRPgyy+TrqZhKEhERLKstBTWrIG77066koYRa5CYWX8ze8vMFpvZldXcf72ZzYsu\nb5vZ6pT7NqfcNyXl9u5m9qqZvWNmD5hZizjfg4hIQzv0UOjTJ0wF3rIl6WoyF1uQmFlTYBxwAtAT\nGGxmPVPbuPtl7t7b3XsDY4CHU+5eV3Gfuw9Iuf2PwPXu3gNYBVwU13sQEYlLSQm8/TY880zSlWQu\nziOSPsBid18S7e8+EThlG+0HA/dv6wnNzIB+wKTopruBgQ1Qq4hIVv3oR7Dzzo1jKnCcQdIFSF3r\ncml021bMbDegOzAt5eZWZlZmZjPNrCIs2gGr3X1TGs85LHp8WXl5eSbvQ0SkwbVoEWZwPf00vPVW\n0tVkJs4gsWpuq2lhgEHAJHffnHJbN3cvBs4GbjCzPevynO5+q7sXu3txhw4d6lK3iEhWXHJJCJQx\nY5KuJDNxBslSoGvK9V2BZTW0HUSVbi13XxZ9XQK8ABwArADamFmzNJ5TRCSndeoUzna/6y5YvbrW\n5jkrziCZDfSIZlm1IITFlKqNzGxvoAh4JeW2IjNrGX3fHugLLHR3B6YDZ0RNhwCTY3wPIiKxKimB\ntWvhzjuTrqT+YguSaBxjFDAVWAQ86O4LzOwaM0udhTUYmBiFRIV9gTIze50QHNe6+8LoviuAn5rZ\nYsKYyYS43oOISNwOPBAOPzx0b23eXHv7XGTe2NYzrkZxcbGXlZUlXYaISLX++U8480yYPBkGDKi9\nfbaY2ZxorHqbdGa7iEjCTj0VunbN36nAChIRkYQ1axb2dZ82DebPT7qaulOQiIjkgKFDYbvt8nMq\nsIJERCQHtGsH554L994LK1cmXU3dKEhERHLE6NGwfj3cfnvSldSNgkREJEd897vQrx+MGwebNtXe\nPlcoSEREckhpKXz4ITzySNKVpE9BIiKSQ046Cbp3D3uV5AsFiYhIDmnaNIyVvPwyvPZa0tWkR0Ei\nIpJjfvxj+Na38ueoREEiIpJj2rSBCy6A+++HTz5JupraKUhERHLQ6NGwcSPcckvSldROQSIikoP2\n3hv694fx40Og5DIFiYhIjiothY8/DqsD5zIFiYhIjjr+eNhrr9wfdFeQiIjkqCZNwg6Ks2bBzJlJ\nV1MzBYmISA4bMgR23DG39ypRkIiI5LDWreGii2DSJPjoo6SrqZ6CREQkx40aFfZzHz8+6UqqpyAR\nEclxe+wBP/xhOKdk/fqkq9magkREJA+UlsKKFeFs91yjIBERyQNHHw3f+U4YdHdPuppvUpCIiOQB\nszAV+PXX4aWXkq7mmxQkIiJ54pxzoG3b3JsKrCAREckT228PF18Mjz4K77+fdDWVFCQiInlk5MjQ\nzTVuXNKVVFKQiIjkka5d4bTT4LbbYO3apKsJFCQiInmmpARWr4a//z3pSgIFiYhInunbFw48MKwK\nnAtTgRUkIiJ5pmIq8MKF8NxzSVejIBERyUuDBkHHjrmxV4mCREQkD7VsCcOHwxNPwOLFydaiIBER\nyVPDh0OzZjB2bLJ1KEhERPJU585w5plwxx2wZk1ydShIRETyWEkJfPEF3HVXcjUoSERE8lifPnDo\noTBmDGzZkkwNChIRkTxXWhoG3J96KpnXjzVIzKy/mb1lZovN7Mpq7r/ezOZFl7fNbHWV+3c0s4/M\nbGzKbS9Ez1nxuI5xvgcRkVx3+umwyy7JTQWOLUjMrCkwDjgB6AkMNrOeqW3c/TJ37+3uvYExwMNV\nnuZ3wIvVPP05FY9z909jKF9EJG80bw4jRsAzz8CiRdl//TiPSPoAi919ibtvBCYCp2yj/WDgv5tI\nmtlBQCfgmRhrFBFpFIYNC+eWJHFUEmeQdAE+TLm+NLptK2a2G9AdmBZdbwL8BfhZDc99Z9St9Wsz\nsxqec5iZlZlZWXl5eX3fg4hIXujQAc4+G+65B1atyu5rxxkk1f2Cr2l5sUHAJHffHF0fATzp7h9W\n0/Ycd/8u8P3ocl51T+jut7p7sbsXd+jQoY6li4jkn5IS+OormDAhu68bZ5AsBbqmXN8VWFZD20Gk\ndGsBhwGjzOw94M/A+WZ2LYC7fxR9/QK4j9CFJiJS8Hr3hiOOCGe6b95ce/uGEmeQzAZ6mFl3M2tB\nCIspVRuZ2d5AEfBKxW3ufo67d3P33YHLgXvc/Uoza2Zm7aPHNQdOBt6M8T2IiOSV0tKwDe+UrX7b\nxie2IHH3TcAoYCqwCHjQ3ReY2TVmNiCl6WBgontaq+q3BKaa2RvAPOAj4LYGLl1EJG8NGADdusGN\nN2bvNS2939/5rbi42MvKypIuQ0QkK667Dn7+c5g3D/bfv/7PY2Zz3L24tnY6s11EpJEZOhS23z57\nU4EVJCIijUxREZx3HvzjH7BiRfyv1yz+lxARkWwrKYFly+Dzz6F9+3hfS0EiItII9eyZvZlb6toS\nEZGMKEhERCQjChIREcmIgkRERDKiIBERkYwoSEREJCMKEhERyYiCREREMlIQizaaWTnwfj0f3h7I\nwiIDdaa66kZ11Y3qqpvGWtdu7l7rzoAFESSZMLOydFa/zDbVVTeqq25UV90Uel3q2hIRkYwoSERE\nJCMKktrdmnQBNVBddaO66kZ8Ac3eAAAGdElEQVR11U1B16UxEhERyYiOSEREJCMKkirM7EdmtsDM\ntphZjbMdzKy/mb1lZovN7Mos1NXWzJ41s3eir0U1tNtsZvOiS2y7EdT2/s2spZk9EN3/qpntHlct\ndazrAjMrT/mMhmahpjvM7FMze7OG+83MbopqfsPMDoy7pjTrOsrMPk/5rH6Tpbq6mtl0M1sU/SyW\nVtMm659ZmnVl/TMzs1ZmNsvMXo/qurqaNvH+PLq7LikXYF9gb+AFoLiGNk2Bd4E9gBbA60DPmOv6\nE3Bl9P2VwB9raPdlFj6jWt8/MAL4W/T9IOCBHKnrAmBslv9PHQEcCLxZw/0nAk8BBhwKvJojdR0F\nPJ7Nzyp63c7AgdH3OwBvV/PvmPXPLM26sv6ZRZ9B6+j75sCrwKFV2sT686gjkircfZG7v1VLsz7A\nYndf4u4bgYnAKTGXdgpwd/T93cDAmF9vW9J5/6n1TgKOMTPLgbqyzt1nAJ9to8kpwD0ezATamFnn\nHKgrEe6+3N1fi77/AlgEdKnSLOufWZp1ZV30GXwZXW0eXaoOfsf686ggqZ8uwIcp15cS/3+oTu6+\nHMJ/aKBjDe1amVmZmc00s7jCJp33/9827r4J+BxoF1M9dakL4PSoO2SSmXWNuaZ0JPH/KV2HRV0m\nT5lZr2y/eNQFcwDhr+xUiX5m26gLEvjMzKypmc0DPgWedfcaP684fh4Lcs92M3sO2Lmau37p7pPT\neYpqbst4+tu26qrD03Rz92Vmtgcwzczmu/u7mdZWRTrvP5bPqBbpvOZjwP3uvsHMhhP+SusXc121\nSeKzSsdrhCUyvjSzE4FHgR7ZenEzaw08BPzE3ddUvbuah2TlM6ulrkQ+M3ffDPQ2szbAI2b2HXdP\nHfuK9fMqyCBx92MzfIqlQOpfsrsCyzJ8zm3WZWafmFlnd18eHcJ/WsNzLIu+LjGzFwh/NTV0kKTz\n/ivaLDWzZsBOxN+NUmtd7r4y5eptwB9jrikdsfx/ylTqL0l3f9LMbjaz9u4e+5pSZtac8Mv6H+7+\ncDVNEvnMaqsryc8ses3V0c99fyA1SGL9eVTXVv3MBnqYWXcza0EYvIpthlRkCjAk+n4IsNWRk5kV\nmVnL6Pv2QF9gYQy1pPP+U+s9A5jm0UhfjGqtq0o/+gBCP3fSpgDnRzORDgU+r+jGTJKZ7VzRj25m\nfQi/L1Zu+1EN8roGTAAWuftfa2iW9c8snbqS+MzMrEN0JIKZbQccC/ynSrN4fx6zObsgHy7AqYT0\n3gB8AkyNbt8FeDKl3YmEWRvvErrE4q6rHfA88E70tW10ezFwe/T994D5hNlK84GLYqxnq/cPXAMM\niL5vBfwTWAzMAvbI0r9fbXX9L7Ag+oymA/tkoab7geXA19H/rYuA4cDw6H4DxkU1z6eG2YIJ1DUq\n5bOaCXwvS3UdTuh2eQOYF11OTPozS7OurH9mwH7A3KiuN4HfVPP/PtafR53ZLiIiGVHXloiIZERB\nIiIiGVGQiIhIRhQkIiKSEQWJiIhkREEi0gDM7MvaW23z8ZOi1Qgws9ZmdouZvRut5jrDzA4xsxbR\n9wV5IrHkLgWJSMKi9ZiauvuS6KbbCWcd93D3XoQVi9t7WIjyeeCsRAoVqYGCRKQBRWdaX2dmb5rZ\nfDM7K7q9SbRcxgIze9zMnjSzM6KHnUO0UoGZ7QkcAvzK3bdAWO7G3Z+I2j4atRfJGTpEFmlYpwG9\ngf2B9sBsM5tBWK5md+C7hJWbFwF3RI/pSzjLHKAXMM/DInzVeRM4OJbKRepJRyQiDetwwurCm939\nE+BFwi/+w4F/uvsWd/+YsDxLhc5AeTpPHgXMRjPboYHrFqk3BYlIw6pps6BtbSK0jrAWEoR1mvY3\ns239bLYE1tejNpFYKEhEGtYM4Kxoo6EOhO1sZwEvEzbUamJmnQhbslZYBHwbwMPeMWXA1SmryPYw\ns1Oi79sB5e7+dbbekEhtFCQiDesRwiqsrwPTgJ9HXVkPEVbYfRO4hbCz3ufRY57gm8EylLDB2WIz\nm0/YN6Vir42jgSfjfQsidaPVf0WyxMxae9g5rx3hKKWvu38c7SExPbpe0yB7xXM8DPzC3d/KQski\nadGsLZHseTzagKgF8LvoSAV3X2dmvyXsq/1BTQ+ONut6VCEiuUZHJCIikhGNkYiISEYUJCIikhEF\niYiIZERBIiIiGVGQiIhIRhQkIiKSkf8P6i1Kj4o9z5sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a14148850>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#需要调优的参数\n",
    "#SVM太慢，每次只调一个参数（这里只调C，penalty为‘l2'）\n",
    "C_s = np.logspace(-1, 3, 5)# logspace(a,b,N)把10的a次方到10的b次方区间分成N份  \n",
    "#penalty_s = ['l1','l2']\n",
    "\n",
    "accuracy_s = []\n",
    "for i, oneC in enumerate(C_s):\n",
    "#    for j, penalty in enumerate(penalty_s):\n",
    "    tmp = fit_grid_point_Linear(oneC, X_train_part, y_train_part, X_val, y_val)\n",
    "    accuracy_s.append(tmp)\n",
    "\n",
    "x_axis = np.log10(C_s)\n",
    "#for j, penalty in enumerate(penalty_s):\n",
    "plt.plot(x_axis, np.array(accuracy_s), 'b-')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'accuracy' )\n",
    "#plt.savefig('SVM_Otto.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.0\n"
     ]
    }
   ],
   "source": [
    "### 最佳超参数\n",
    "index = np.argmax(accuracy_s, axis=None)\n",
    "Best_C = C_s[ index ]\n",
    "\n",
    "print(Best_C)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 找到最佳参数后，用全体训练数据训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# SVC训练SVC，支持概率输出\n",
    "Best_C = 100\n",
    "\n",
    "SVC3 = LinearSVC(C = Best_C)\n",
    "SVC3.fit(X_train, y_train)\n",
    "\n",
    "#保持模型，用于后续测试\n",
    "import cPickle\n",
    "cPickle.dump(SVC3, open(\"Otto_LinearSVC.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
